"""
auth:xgt-python
datetime:2021/11/5
爬取付费文档网站
"""

import requests # 数据请求模块
import re # 正则表达式模块
# 可以直接提取字符串数据 response.text 就是html字符串数据
import parsel #数据解析模块  转换成课解析的对象 selector
import os #文件操作模块
import pdfkit # 转成pdf



html_str  = """
<!doctype html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Document</title>
</head>
<body>
{article}
</body>
</html>    
"""
def change_title(name):
      new_name = re.sub(r'[\\\/\:\*\?\"\<\>\|]','_',name)
      return new_name





# htmltopdf 的软件
pdf_filename = 'pdf\\' # 定义文件夹名字
# 判断 如果没有这个文件夹的话
if not os.path.exists(pdf_filename):
    os.mkdir(pdf_filename)  # 创建这个文件夹

html_filename = 'html\\'
if not os.path.exists(html_filename):
    os.mkdir(html_filename)  # 创建这个文件夹

for page in range(2,65):

    url = f'https://www.chinawenwang.com/zlist-66-{page}.html'

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.54 Safari/537.36'
    }
    response = requests.get(url=url,headers=headers)
    # print(response.text)
    # 解析数据 用re正则表达式提取文章url地址 .*? 可以匹配任意字符 除了(\n) 如果需要匹配\n则需要加上re.S
    # .*? 各代表的是啥意思？属于正则表达式里面的元字符 . 可以匹配任意字符除了(\n) * 匹配前一个字符0个或者无限个 ? 非贪婪匹配模式
    href = re.findall('<h2><a href="(.*?)" class="juhe-page-left-div-link">',response.text)
    for link in href:
        response_1 = requests.get(url=link,headers=headers)
        selector = parsel.Selector(response_1.text)
        title = selector.css('.content-page-header-div h1::text').get()
        name = change_title(title)
        # 我要获取包含html标签的
        content = selector.css('.content-page-main-content-div').get()
        article = html_str.format(article=content)
        pdf_path = pdf_filename + name + '.pdf' #html文件的路径/文件名/后缀
        html_path = html_filename + name + '.html' #html文件的路径/文件名/后缀
        try:
            with open(html_path,mode='w',encoding='utf-8') as f:
                f.write(article)
            config = pdfkit.configuration(wkhtmltopdf=r'C:\Program Files\wkhtmltopdf\bin\wkhtmltopdf.exe')
            pdfkit.from_file(html_path,pdf_path,configuration=config)
            print('保存成功',title)
        except:
            pass

